package com.wontube.smr.bo.sites.hulu;

import java.io.IOException;
import java.io.InputStreamReader;
import java.math.BigDecimal;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.UUID;

import org.apache.ibatis.session.RowBounds;
import org.apache.ibatis.session.SqlSession;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.JsonParser;
import com.wontube.client.extension.utils.DateUtils;
import com.wontube.client.extension.utils.StringUtils;
import com.wontube.core.dao.mybatis.QueryExample;
import com.wontube.core.pub.AppEnvironment;
import com.wontube.smr.bo.constants.SiteConstants;
import com.wontube.smr.bo.constants.StandarAreas;
import com.wontube.smr.bo.constants.StandarMovieGenres;
import com.wontube.smr.client.model.po.Movie;
import com.wontube.smr.client.model.po.MovieGenre;
import com.wontube.smr.client.model.po.MovieWeek;
import com.wontube.smr.client.model.po.PopularEpisode;
import com.wontube.smr.client.model.po.RecentEpisode;
import com.wontube.smr.client.model.po.RecentMovie;
import com.wontube.smr.client.model.po.RecentShow;
import com.wontube.smr.client.model.po.TVEpisode;
import com.wontube.smr.client.model.po.TVSeason;
import com.wontube.smr.client.model.po.TVShow;
import com.wontube.smr.client.model.po.TVShowGenre;
import com.wontube.smr.dao.mapper.MovieGenreMapper;
import com.wontube.smr.dao.mapper.MovieMapper;
import com.wontube.smr.dao.mapper.MovieWeekMapper;
import com.wontube.smr.dao.mapper.PopularEpisodeMapper;
import com.wontube.smr.dao.mapper.RecentEpisodeMapper;
import com.wontube.smr.dao.mapper.RecentMovieMapper;
import com.wontube.smr.dao.mapper.RecentShowMapper;
import com.wontube.smr.dao.mapper.TVEpisodeMapper;
import com.wontube.smr.dao.mapper.TVSeasonMapper;
import com.wontube.smr.dao.mapper.TVShowGenreMapper;
import com.wontube.smr.dao.mapper.TVShowMapper;
import com.wontube.smr.pub.DefaultDBHelper;

/**
 * @author yuechao
 * 
 */
public final class HuluObtainBO {
	// eg.
	// http://www.hulu.com/browse/popular/episodes/1?timeframe=today&has_captions=1&has_free=1
	private static final String popularTVUrlFormat = "http://www.hulu.com/browse/popular/episodes/{}?timeframe=today&has_free=1";
	private static final String tvItemUrlFormat = "http://www.hulu.com/videos/slider?classic_sort=asc&items_per_page=5&page={}&show_id={}&show_placeholders=1&sort=original_premiere_date&type=episode";
	// eg.
	// http://www.hulu.com/browse/recent/tv/1?has_free=1
	private static final String recentTVUrlFormat = "http://www.hulu.com/browse/recent/tv/{}?has_free=1";

	// eg.
	// http://www.hulu.com/popular/episodes?h=1&has_free=1&page=1&timeframe=today
	private static final String popularEpisodeUrlFormat = "http://www.hulu.com/popular/episodes?h=1&has_free=1&page={}&timeframe=today";
	// eg.
	// http://www.hulu.com/recent/episodes?h=1&has_free=1&page=2
	private static final String recentEpisodeUrlFormat = "http://www.hulu.com/recent/episodes?h=1&has_free=1&page={}";

	// eg.
	// http://www.hulu.com/popular/feature_films?h=23&has_free=1&page=1&timeframe=today
	private static final String popularMovieUrlFormat = "http://www.hulu.com/popular/feature_films?h=23&has_free=1&page={}&timeframe=today";
	
	// eg.
	// http://www.hulu.com/popular/feature_films?h=23&has_free=1&page=1&timeframe=this_week
	private static final String popularWeekMovieUrlFormat = "http://www.hulu.com/popular/feature_films?h=23&has_free=1&page={}&timeframe=this_week";
	
	// eg.
	// http://www.hulu.com/recent/feature_films?h=0&has_free=1&page=1
	private static final String recentMovieUrlFormat = "http://www.hulu.com/recent/feature_films?h=0&has_free=1&page={}";

	// http://www.hulu.com/recent/games?h=1&has_free=1&page=1
	private static final String popularGamesUrlFormat = "http://www.hulu.com/popular/games?h=2&has_free=1&page={}";
	// eg.
	// http://www.hulu.com/recent/games?h=1&has_free=1&page=1
	private static final String recentGamesUrlFormat = "http://www.hulu.com/recent/games?h=2&has_free=1&page={}";
	
	private static final String REFERER = "http://www.hulu.com/";
	private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; rv:13.0) Gecko/20100101 Firefox/13.0.1";
	private static final int TIMEOUT = 30 * 1000;

	private static final Logger logger = LoggerFactory.getLogger(HuluObtainBO.class);	

	private HuluObtainBO() {
	}

	/**
	 * save tv and tv episodes
	 * @return
	 * @throws IOException
	 */
	public static int saveTV() throws IOException {
		int count = 0;
		// 先假定只有一页
		int pageCount = 1;
		for (int page = 1; page <= pageCount; page++) {
			String url = StringUtils.format(popularTVUrlFormat, page);
			Document doc = getDocument(url);
			if(doc == null){
				continue;
			}
			
			if (page == 1) { // 修正页数
				String str = doc.select("div.pagination-bottom > div.page > ul li.total > a").eq(0).text();
				if ( ! "".equals(str)) {
					pageCount = Integer.parseInt(str);
				}
			}

			List<TVShow> list = obtainTV(doc);
			if (list.size() < 1) {
				continue;
			}

			for (TVShow tvShow : list) {
				List<TVEpisode> listEpisode = obtainTVEpisodes(tvShow);
				if (listEpisode.size() < 1) {
					continue;
				}		
				
				Date version = DateUtils.getNow();				
				tvShow.setVersion(version);
				String showId = tvShow.getId();
				tvShow.setSortIndex(++count);
				
				//修正Hulu TV genre
				tvShow.setGenre(HuluTVGenres.remendAllGenres(tvShow.getGenre()));				
				String genreNames = tvShow.getGenre();				
				
				SqlSession session = DefaultDBHelper.getSqlSession();				
				try {					
					TVShowMapper mapper = session.getMapper(TVShowMapper.class);
					TVShowGenreMapper genreMapper = session.getMapper(TVShowGenreMapper.class);
					
					// 主表					
					TVShow tmpTv = mapper.selectByPrimaryKey(showId);									
					if (tmpTv != null) { // show已存在						
						tvShow.setScore(tmpTv.getScore()); // important, 不更新 score
						tvShow.setAirBeginDate(tmpTv.getAirBeginDate()); // important, 不更新publish_time
						tvShow.setAirEndDate(tmpTv.getAirEndDate()); // important, 不更新publish_time
						
						mapper.updateByPrimaryKey(tvShow);
					} else { // 不存在，插入
						mapper.insert(tvShow);
					}
					
					// genre 是否已经关联？
					if(genreNames != null){
						String[] genres = genreNames.split("\\s*,\\s*");
						for (String genre : genres) {
							QueryExample genreEx = new QueryExample();
							genreEx.or().addCriterion("show_id = ", showId)
								.addCriterion("genre_name = ", genre);		
							
							if (genreMapper.countByExample(genreEx) == 0) { // genre 未关联
								TVShowGenre tvShowGenre = new TVShowGenre();
								tvShowGenre.setId(UUID.randomUUID().toString());
								tvShowGenre.setShowId(showId);
								tvShowGenre.setGenreName(genre);
								genreMapper.insert(tvShowGenre);
							}
						}
					}			
					
					// 插入明细
					TVEpisodeMapper episodeMapper = session.getMapper(TVEpisodeMapper.class);
					for (TVEpisode episode : listEpisode) {
						String epId = episode.getId();					
						episode.setVersion(version);
						if(episodeMapper.selectByPrimaryKey(epId) != null){
							episodeMapper.updateByPrimaryKey(episode);
						} else { // 不存在，插入
							episodeMapper.insert(episode);
						}
					}
					
					// 统计seasons
					QueryExample seasonEx = new QueryExample();
					seasonEx.or().addCriterion("tv_season.show_id = ", showId);
					List<Integer> seasons = episodeMapper.selectSeasons(showId);
					TVSeasonMapper seasonMapper = session.getMapper(TVSeasonMapper.class);
					seasonMapper.deleteByExample(seasonEx);
					for (Integer seasonNo : seasons) {
						seasonEx.clear();
						seasonEx.or().addCriterion("tv_season.show_id = ", showId)
							.addCriterion("tv_season.season_no = ", seasonNo);
						if (seasonMapper.countByExample(seasonEx) < 1) { // 不存在
							TVSeason season = new TVSeason();
							season.setId(UUID.randomUUID().toString());
							season.setShowId(showId);
							season.setSeasonNo(seasonNo);
							season.setVersion(Calendar.getInstance().getTime());
							seasonMapper.insert(season);
						}
					}
					
					session.commit();
				} finally {
					session.close();
				}
			}
		}
		return count;
	}

	/**
	 * 
	 * 为tv评分，用最后一个有分数的单集的评分作为tvshow的评分
	 * 
	 * 从tv_episode的数据更新tv_show里的score, airStartDate, airEndDate三个字段.
	 * @return
	 */
	public static int scoreAndTimeTV() {
		int count = 0;
		int index = 0;
		QueryExample ex = new QueryExample();
		ex.or().addCriterion("tv_show.website=", SiteConstants.WEBSITE_HULU);

		while (true) {
			RowBounds rowBounds = new RowBounds(index, 5);
			SqlSession session = DefaultDBHelper.getBatchSqlSession();
			try {
				TVShowMapper mapper = session.getMapper(TVShowMapper.class);
				TVEpisodeMapper tvEpisodeMapper = session.getMapper(TVEpisodeMapper.class);
				List<TVShow> listTVShow = mapper.selectByExample(ex, rowBounds);
				
				if (listTVShow.size() < 1) {
					break;
				}
				
				for (TVShow tvShow : listTVShow) {
					QueryExample tvShowEx = new QueryExample();
					tvShowEx.or().addCriterion("tv_episode.show_id = ", tvShow.getId())
						.addCriterion("tv_episode.air_date is not null");
					tvShowEx.setOrderByClause("tv_episode.part desc");
					RowBounds tvShowBounds = new RowBounds(0, 1);
					List<TVEpisode> listEpisode = tvEpisodeMapper.selectWithPage(tvShowEx, tvShowBounds);
					if (listEpisode.size() < 1) {
						continue;
					}
	
					Date airEndDate = listEpisode.get(0).getAirDate();
					if (airEndDate != null) {
						tvShow.setAirEndDate(airEndDate);
					}
					
					tvShowEx.setOrderByClause("tv_episode.part asc");
					listEpisode = tvEpisodeMapper.selectWithPage(tvShowEx, tvShowBounds);
					if (listEpisode.size() < 1) {
						continue;
					}
	
					Date airBeginDate = listEpisode.get(0).getAirDate();
					if (airEndDate != null) {
						tvShow.setAirBeginDate(airBeginDate);
					}
					
					tvShowEx.clear();
					tvShowEx.or().addCriterion("tv_episode.show_id = ", tvShow.getId())
						.addCriterion("tv_episode.score > 0");
					tvShowEx.setOrderByClause("tv_episode.part desc");
					listEpisode = tvEpisodeMapper.selectWithPage(tvShowEx, tvShowBounds);
					if (listEpisode.size() < 1) {
						continue;
					}
					
					BigDecimal score = listEpisode.get(0).getScore();
					tvShow.setScore(score);

					mapper.updateByPrimaryKey(tvShow);
					count++;
				}
				session.commit();
				index += listTVShow.size();
			} finally {
				session.close();
			}
		}
		
		return count;
	}
	
	
	/**
	 * save or update to recent_show table
	 * @return
	 * @throws IOException
	 */
	public static int saveRecentTV() throws IOException {
		int count = 0;
		// 先假定只有一页
		int pageCount = 1;
		for (int page = 1; page <= pageCount; page++) {
			String url = StringUtils.format(recentTVUrlFormat, page);
			Document doc = getDocument(url);
			if(doc == null){
				continue;
			}
						
			if (page == 1) { // 修正页数
				String str = doc.select("div.pagination-bottom > div.page > ul li.total > a").eq(0).text();
				if (!"".equals(str)) {
					pageCount = Integer.parseInt(str);
					if (pageCount > 20) { // recent最多抓20页、480项
						pageCount = 20;
					}
				}
			}

			//只取tvshow id
			List<String> showIds = obtainTVShowIds(doc);
			if (showIds.size() < 1) {
				continue;
			}

			QueryExample ex = new QueryExample();
			SqlSession session = DefaultDBHelper.getSqlSession();
			Date version = DateUtils.getNow();
			try {
				RecentShowMapper recentShowMapper = session.getMapper(RecentShowMapper.class);
				TVShowMapper tvShowMapper = session.getMapper(TVShowMapper.class);

				for (String showId : showIds) {
					TVShow temp = tvShowMapper.selectByPrimaryKey(showId);
					if(temp == null){
						// 不在所有的项目中
						continue;
					}
					
					RecentShow recentShow = new RecentShow();					
					recentShow.setShowId(showId);
					recentShow.setSortIndex(++count);
					recentShow.setVersion(version);

					ex.clear();
					ex.or().addCriterion("show_id = ", showId);
					List<RecentShow> recentShows = recentShowMapper.selectByExample(ex);
					if (recentShows.size() > 0) { // 已经在表中
						recentShow.setId(recentShows.get(0).getId());
						recentShowMapper.updateByPrimaryKey(recentShow);
					} else { // 不存在
						recentShow.setId(UUID.randomUUID().toString());
						recentShowMapper.insert(recentShow);
					}
				}
				session.commit();
			} finally {
				session.close();
			}
		}
		return count;
	}

	private static List<String> obtainTVShowIds(Document doc) {
		List<String> showIds = new ArrayList<String>();
		Elements elements = doc.select("#results_list > tbody tr > td > h3 > div.play-button-hover > a");
		for (Element ele : elements) {
			String beaconid = ele.attr("beaconid");
			if( ! "".equals(beaconid)){
				showIds.add(SiteConstants.ID_PREFIX_HULU + beaconid);
			}
		}
		
		return showIds;
	}	

	/**
	 * save to popular_episode
	 * @return
	 * @throws IOException
	 */
	public static int savePopularEpisodes() throws IOException {
		int count = 0;
		// 先假定只有一页
		int pageCount = 1;
		for (int page = 1; page <= pageCount; page++) {
			String url = StringUtils.format(popularEpisodeUrlFormat, page);
			Document doc = getDocument(url);
			if(doc == null){
				continue;
			}
			
			if (page == 1) { // 修正页数
				String str = doc.select("div.pagination-bottom > div.page > ul li.total > a").eq(0).text();
				if ( ! "".equals(str)) {
					pageCount = Integer.parseInt(str);
					if (pageCount > 200) { // popular episodes 最多抓100页、4800项
						pageCount = 200;
					}
				}
			}

			//只取episode videoId
			List<String> videoIds = obtainEpisodeVideoIds(doc);			
			if (videoIds.size() < 1) {
				continue;
			}

			Date version = DateUtils.getNow();
			QueryExample ex = new QueryExample();
			SqlSession session = DefaultDBHelper.getBatchSqlSession();
			try {
				PopularEpisodeMapper popularEpisodeMapper = session.getMapper(PopularEpisodeMapper.class);
				TVEpisodeMapper tvEpisodeMapper = session.getMapper(TVEpisodeMapper.class);

				for (String videoId : videoIds) {
					ex.clear();
					ex.or().addCriterion("tv_episode.video_id=", videoId)
						.addCriterion("tv_episode.website = ", SiteConstants.WEBSITE_HULU);
					List<TVEpisode> episodes = tvEpisodeMapper.selectByExample(ex);
					if (episodes.size() < 1) { // 不在所有的项目中
						continue;
					}

					String episodeId = episodes.get(0).getId();
					PopularEpisode popularEpisode = new PopularEpisode();					
					popularEpisode.setEpisodeId(episodeId);
					popularEpisode.setSortIndex(++count);
					popularEpisode.setVersion(version);

					ex.clear();
					ex.or().addCriterion("episode_id = ", episodeId);
					List<PopularEpisode> popularEpisodes = popularEpisodeMapper.selectByExample(ex);
					if (popularEpisodes.size() > 0) { // 已经在表中
						popularEpisode.setId(popularEpisodes.get(0).getId());
						popularEpisodeMapper.updateByPrimaryKey(popularEpisode);
					} else { // 不存在
						popularEpisode.setId(UUID.randomUUID().toString());
						popularEpisodeMapper.insert(popularEpisode);
					}
				}
				session.commit();
			} finally {
				session.close();
			}
		}
		return count;
	}
	

	private static List<String> obtainEpisodeVideoIds(Document doc) {
		List<String> videoIds = new ArrayList<String>();
		Elements elements = doc.select("#results_list > tbody tr > td > div.home-thumb > div.home-play-container > span.play-button-hover > a");
		for (Element ele : elements) {
			String beaconid = ele.attr("beaconid");
			if( ! "".equals(beaconid)){
				videoIds.add(beaconid);
			}
		}
		return videoIds;
	}

	/**
	 * save to recent_episode
	 * @return
	 * @throws IOException
	 */
	public static int saveRecentEpisodes() throws IOException {
		int count = 0;
		// 先假定只有一页
		int pageCount = 1;
		for (int page = 1; page <= pageCount; page++) {
			String url = StringUtils.format(recentEpisodeUrlFormat, page);
			Document doc = getDocument(url);
			if(doc == null){
				continue;
			}
			
			if (page == 1) { // 修正页数
				String str = doc.select("div.pagination-bottom > div.page > ul li.total > a").eq(0).text();
				if (!"".equals(str)) {
					pageCount = Integer.parseInt(str);
					if (pageCount > 20) { // recent最多抓20页、480项
						pageCount = 20;
					}
				}
			}

			//只取episode videoId
			List<String> videoIds = obtainEpisodeVideoIds(doc);			
			if (videoIds.size() < 1) {
				continue;
			}

			Date version = DateUtils.getNow();
			QueryExample ex = new QueryExample();
			SqlSession session = DefaultDBHelper.getBatchSqlSession();
						
			try {
				RecentEpisodeMapper recentEpisodeMapper = session.getMapper(RecentEpisodeMapper.class);
				TVEpisodeMapper tvEpisodeMapper = session.getMapper(TVEpisodeMapper.class);
				
				for (String videoId: videoIds) {
					ex.clear();
					ex.or().addCriterion("tv_episode.video_id=", videoId)
						.addCriterion("tv_episode.website = ", SiteConstants.WEBSITE_HULU);
					List<TVEpisode> episodes = tvEpisodeMapper.selectByExample(ex);
					if (episodes.size() < 1) { // 不在所有的项目中
						continue;
					}

					String episodeId = episodes.get(0).getId();
					RecentEpisode recentEpisode = new RecentEpisode();					
					recentEpisode.setEpisodeId(episodeId);
					recentEpisode.setSortIndex(++count);
					recentEpisode.setVersion(version);

					ex.clear();
					ex.or().addCriterion("episode_id=", episodeId);
					List<RecentEpisode> recentEpisodes = recentEpisodeMapper.selectByExample(ex);
					if (recentEpisodes.size() > 0) { // 已经在表中
						recentEpisode.setId(recentEpisodes.get(0).getId());
						recentEpisodeMapper.updateByPrimaryKey(recentEpisode);
					} else { // 不存在
						recentEpisode.setId(UUID.randomUUID().toString());
						recentEpisodeMapper.insert(recentEpisode);
					}
				}
				session.commit();
			} finally {
				session.close();
			}
		}

		return count;
	}

	/**
	 * 
	 * @return
	 * @throws IOException
	 */
	public static int saveMovies() throws IOException {		
		int count = 0;
		// 先假定只有一页
		int pageCount = 1;
		for (int page = 1; page <= pageCount; page++) {
			String url = StringUtils.format(popularMovieUrlFormat, page);
			Document doc = getDocument(url);
			if(doc == null){
				continue;
			}
			
			if (page == 1) { // 修正页数
				String str = doc.select("div.pagination-bottom > div.page > ul li.total > a").eq(0).text();
				if ( ! "".equals(str)) {
					pageCount = Integer.parseInt(str);
				}
			}

			List<Movie> listMovie = obtainMovies(doc, Movie.TYPE_MOVIE);
			if (listMovie.size() < 1) {
				continue;
			}

			SqlSession session = DefaultDBHelper.getBatchSqlSession();			
			Date now = DateUtils.getNow();
			try {
				MovieMapper mapper = session.getMapper(MovieMapper.class);
				MovieGenreMapper genreMapper = session.getMapper(MovieGenreMapper.class);
				
				for (Movie movie : listMovie) {			
					String movieId = movie.getId();
					// 修正 MovieType
					movie.setMovieType(Movie.TYPE_MOVIE);
					movie.setSortIndex(++count);		
					movie.setVersion(now);
					// 修正 Hulu genre
					movie.setGenre(HuluMovieGenres.remendAllGenres(movie.getGenre()));
					String genreNames = movie.getGenre();
					
					Movie temp = mapper.selectByPrimaryKey(movieId);							
					if(temp != null){
						//保留大图					
						movie.setBannerImgUrl(temp.getBannerImgUrl());
						mapper.updateByPrimaryKey(movie);						
					}else{
						mapper.insert(movie);
					}
					
					if(genreNames != null){
						String[] genres = genreNames.split("\\s*,\\s*");
						for (String genre : genres) {
							QueryExample genreEx = new QueryExample();
							genreEx.or().addCriterion("movie_id = ", movieId)
								.addCriterion("genre_name = ", genre);
							
							//关联genre
							if (genreMapper.countByExample(genreEx) == 0) {
								MovieGenre movieGenre = new MovieGenre();
								movieGenre.setId(UUID.randomUUID().toString());
								movieGenre.setMovieId(movieId);
								movieGenre.setGenreName(genre);
								
								genreMapper.insert(movieGenre);
							}
						}
					}					
				}
				session.commit();
			} finally {
				session.close();
			}
		}
		
		return count;
	}

	/**
	 * save to recent_movie
	 * @return
	 * @throws IOException
	 */
	public static final int saveRecentMovies() throws IOException {
		return saveRecentMovies(recentMovieUrlFormat);
	}
	
	/**
	 * save to movie_week
	 * @return
	 * @throws IOException
	 */
	public static final int savePopularWeekMovies() throws IOException {
		int count = 0;
		// 先假定只有一页
		int pageCount = 1;
		for (int page = 1; page <= pageCount; page++) {
			String url = StringUtils.format(popularWeekMovieUrlFormat, page);
			Document doc = getDocument(url);
			if(doc == null){
				continue;
			}
			
			if (page == 1) { // 修正页数
				String str = doc.select("div.pagination-bottom > div.page > ul li.total > a").eq(0).text();
				if (!"".equals(str)) {
					pageCount = Integer.parseInt(str);
					if (pageCount > 20) { // recent 最多抓20页，480项
						pageCount = 20;
					}
				}
			}

			//只取movieId
			List<String> movieIds = obtainMovieIds(doc);
			if (movieIds.size() < 1) {
				continue;
			}

			QueryExample ex = new QueryExample();
			SqlSession session = DefaultDBHelper.getBatchSqlSession();			
			Date version = DateUtils.getNow();
			try {
				MovieMapper mapper = session.getMapper(MovieMapper.class);
				MovieWeekMapper weekMapper = session.getMapper(MovieWeekMapper.class);
				
				for (String movieId: movieIds) {
					Movie temp = mapper.selectByPrimaryKey(movieId);
					if(temp == null){
						 // 不在所有的项目中
						continue;
					}
					
					MovieWeek movieWeek = new MovieWeek();					
					movieWeek.setMovieId(movieId);
					movieWeek.setSortIndex(++count);
					movieWeek.setVersion(version);

					ex.clear();
					ex.or().addCriterion("movie_id = ", movieId);
					List<MovieWeek> movieWeeks = weekMapper.selectByExample(ex);
					if (movieWeeks.size() > 0) { // 已存在
						movieWeek.setId(movieWeeks.get(0).getId());
						weekMapper.updateByPrimaryKey(movieWeek);
					} else { // 不存在
						movieWeek.setId(UUID.randomUUID().toString());
						weekMapper.insert(movieWeek);
					}
				}
				session.commit();
			} finally {
				session.close();
			}
		}
		
		return count;
	}
	
	/**
	 * save to movie with movie_type='Games'
	 * @return
	 * @throws IOException
	 */
	public static int savePopularGames() throws IOException {		
		int count = 0;
		// 先假定只有一页
		int pageCount = 1;
		for (int page = 1; page <= pageCount; page++) {
			String url = StringUtils.format(popularGamesUrlFormat, page);
			Document doc = getDocument(url);
			if(doc == null){
				continue;
			}
			
			if (page == 1) { // 修正页数
				String str = doc.select("div.pagination-bottom > div.page > ul li.total > a").eq(0).text();
				if (!"".equals(str)) {
					pageCount = Integer.parseInt(str);
				}
			}

			List<Movie> listMovie = obtainMovies(doc, Movie.TYPE_GAMES);
			if (listMovie.size() < 1) {
				continue;
			}

			SqlSession session = DefaultDBHelper.getBatchSqlSession();
			Date version = DateUtils.getNow();
			try {
				MovieMapper mapper = session.getMapper(MovieMapper.class);
				MovieGenreMapper genreMapper = session.getMapper(MovieGenreMapper.class);
				
				for (Movie movie : listMovie) {				
					// 修正genre
					movie.setGenre(StandarMovieGenres.SPORTS);
					movie.setVersion(version);
					movie.setSortIndex(++count);
					
					String movieId = movie.getId();
					Movie temp = mapper.selectByPrimaryKey(movieId);
					
					String genreName = movie.getGenre();
					boolean isGenreRefed = false;
					
					if (temp != null) { // 已存在						
						QueryExample genreEx = new QueryExample();
						genreEx.or().addCriterion("movie_id = ", movieId)
							.addCriterion("genre_name = ", genreName);
						int genreRefCount = genreMapper.countByExample(genreEx);
						isGenreRefed = genreRefCount > 0;
						
						if ( ! isGenreRefed) {
							movie.setGenre(temp.getGenre() + "," + genreName);
						}
						// 大图抓不到，用原来的
						movie.setBannerImgUrl(temp.getBannerImgUrl());
						mapper.updateByPrimaryKey(movie);
					} else { // 不存在
						mapper.insert(movie);
					}
					
					if ( ! isGenreRefed) {
						MovieGenre movieGenre = new MovieGenre();
						movieGenre.setId(UUID.randomUUID().toString());
						movieGenre.setMovieId(movieId);
						movieGenre.setGenreName(genreName);
						genreMapper.insert(movieGenre);
					}
				}
				session.commit();
			} finally {
				session.close();
			}
		}
		
		return count;
	}
	
	public static int saveRecentGames() throws IOException {
		return saveRecentMovies(recentGamesUrlFormat);
	}
	
	/**
	 * 更新视频banner
	 * @param movie 视频对象
	 * @return
	 */
	public static List<Movie> updateVideoBanner(Movie movie) {
		List<Movie> movies = new ArrayList<Movie>();
		SqlSession session = DefaultDBHelper.getBatchSqlSession();
		try {
			MovieMapper mapper = session.getMapper(MovieMapper.class);
			Movie po=mapper.selectByPrimaryKey(movie.getId());
			po.setBannerImgUrl(movie.getBannerImgUrl());
			mapper.updateByPrimaryKey(po);
			session.commit();
		} finally {
			session.close();
		}
		return movies;
	}
	
	private static final int saveRecentMovies(String urlFormat) throws IOException {
		int count = 0;
		// 先假定只有一页
		int pageCount = 1;
		for (int page = 1; page <= pageCount; page++) {
			String url = StringUtils.format(urlFormat, page);
			Document doc = getDocument(url);
			if(doc == null){
				continue;
			}
			
			if (page == 1) { // 修正页数
				String str = doc.select("div.pagination-bottom > div.page > ul li.total > a").eq(0).text();
				if (!"".equals(str)) {
					pageCount = Integer.parseInt(str);
					if (pageCount > 20) { // recent 最多抓20页，480项
						pageCount = 20;
					}
				}
			}

			//只取movie id就足够了			
			List<String> movieIds = obtainMovieIds(doc);
			if (movieIds.size() < 1) {
				continue;
			}

			QueryExample ex = new QueryExample();
			SqlSession session = DefaultDBHelper.getBatchSqlSession();			
			Date version = DateUtils.getNow();
			try {
				MovieMapper mapper = session.getMapper(MovieMapper.class);
				RecentMovieMapper recentMoveMapper = session.getMapper(RecentMovieMapper.class);
				
				for (String movieId : movieIds) {	
					
					Movie temp = mapper.selectByPrimaryKey(movieId);
					if(temp == null){
						// 不在所有的项目中
						continue;
					}
					
					RecentMovie recentMovie = new RecentMovie();					
					recentMovie.setMovieId(movieId);
					recentMovie.setSortIndex(++count);
					recentMovie.setVersion(version);

					ex.clear();
					ex.or().addCriterion("movie_id = ", movieId);
					List<RecentMovie> recentMovies = recentMoveMapper.selectByExample(ex);
					if (recentMovies.size() > 0) { // 已存在
						recentMovie.setId(recentMovies.get(0).getId());
						recentMoveMapper.updateByPrimaryKey(recentMovie);
					} else { // 不存在
						recentMovie.setId(UUID.randomUUID().toString());
						recentMoveMapper.insert(recentMovie);
					}
				}
				session.commit();
			} finally {
				session.close();
			}
		}
		return count;
	}
	
	private static List<String> obtainMovieIds(Document doc) {
		List<String> ids = new ArrayList<String>();
		Elements elements = doc.select("#results_list > tbody tr > td > div.home-thumb > div.home-play-container > span.play-button-hover > a");
		for (Element as : elements) {			
			String beaconid = as.attr("beaconid");			
			if( ! "".equals(beaconid)){
				ids.add(SiteConstants.ID_PREFIX_HULU + beaconid);
			}
		}		
		
		return ids;
	}

	private static List<TVShow> obtainTV(Document doc) throws IOException {
		List<TVShow> listResult = new ArrayList<TVShow>();

		final String baseUrl = "http://www.hulu.com/";
		Elements elements = doc.select("#results_list > tbody tr > td ");
		for (int i = 0; i < elements.size(); i++) {
			Element td = elements.get(i);
			String itemUrl = td.select("div.show-title-container > a").attr("href");

			if (itemUrl.length() > baseUrl.length()) {
				String infoUrl = "http://www.hulu.com/shows/info/" + itemUrl.substring(baseUrl.length());
				HttpURLConnection infoConn = (HttpURLConnection) new URL(infoUrl).openConnection();
				infoConn.connect();

				JsonObject jsonObject = null;
				
				
				InputStreamReader reader = new InputStreamReader(infoConn.getInputStream(), AppEnvironment.charsetName);
				try {
					JsonParser parser = new JsonParser();
					jsonObject = (JsonObject) parser.parse(reader);
				} catch (JsonParseException ex) {
					logger.warn(ex.toString());
				} finally {
					reader.close();
					infoConn.disconnect();
				}

				if (jsonObject != null) {
					TVShow item = new TVShow();										
					item.setScore(BigDecimal.ZERO);
					item.setAirBeginDate(null);
					item.setAirEndDate(null);
					item.setWebsite(SiteConstants.WEBSITE_HULU);
					item.setArea(StandarAreas.AMERICA);
					item.setItemUrl(itemUrl);

					JsonElement name = jsonObject.get("name");
					item.setName(name == null || name.isJsonNull() ? "" : name.getAsString());
					//首字母
					item.setTitleInitial(StringUtils.getInitial(item.getName()));

					JsonElement description = jsonObject.get("description");
					item.setDescription(description == null || description.isJsonNull() ? "" : description.getAsString());

					JsonElement thumbnailUrl = jsonObject.get("thumbnail_url");
					item.setThumbnail(thumbnailUrl == null || thumbnailUrl.isJsonNull() ? "" : thumbnailUrl.getAsString());

					JsonElement id = jsonObject.get("id");
					item.setVideoId(id == null || id.isJsonNull() ? "" : id.getAsString());

					// "hl" + videoid
					item.setId(SiteConstants.ID_PREFIX_HULU + item.getVideoId());
					
					JsonElement seasonsCount = jsonObject.get("seasons_count");
					item.setSeasonCount(seasonsCount == null || seasonsCount.isJsonNull() ? 1 : seasonsCount.getAsInt());
					
					JsonElement episodeCount = jsonObject.get("episodes_count");
					item.setEpisodeCnt(episodeCount == null || episodeCount.isJsonNull() ? 0 : episodeCount.getAsInt());

					JsonElement genre = jsonObject.get("channel");
					item.setGenre(genre == null || genre.isJsonNull() ? "" : genre.getAsString());
					String genreName = item.getGenre();
					if(genreName != null){
						item.setGenre(genreName.split("~")[0]);
					}

					listResult.add(item);
				}
			}
		}

		return listResult;
	}

	private static List<TVEpisode> obtainTVEpisodes(final TVShow tvShow) throws IOException {
		List<TVEpisode> listResult = new ArrayList<TVEpisode>();
		final SimpleDateFormat airDateFormat = new SimpleDateFormat("MM/dd/yyyy");
		
		String id = tvShow.getVideoId();
		int pageIndex = 1;

		while (listResult.size() < tvShow.getEpisodeCnt()) {
			String url = StringUtils.format(tvItemUrlFormat, pageIndex, id);
			pageIndex++;
			Document doc = getDocument(url);
			if(doc == null){
				continue;
			}

			Elements elements = doc.select("li");
			if (elements.size() < 1) {
				break;
			}

			for (int i = 0; i < elements.size(); i++) {
				TVEpisode episode = new TVEpisode();				
				episode.setShowId(tvShow.getId());
				episode.setWebsite(SiteConstants.WEBSITE_HULU);			

				Element li = elements.get(i);
				Element link = li.select("span.play-button-hover > a.beaconid").first();

				String beaconid = link.attr("beaconid");
				if ("".equals(beaconid)) {
					continue;
				}

				String href = link.attr("href");
				if ("".equals(href)) {
					continue;
				}
				
				episode.setVideoId(beaconid);

				String infoUrl = "http://www.hulu.com/videos/info/" + beaconid;
				HttpURLConnection infoConn = (HttpURLConnection) new URL(infoUrl).openConnection();
				infoConn.connect();

				JsonObject jsonObject = null;
				InputStreamReader reader = new InputStreamReader(infoConn.getInputStream(), AppEnvironment.charsetName);
				try {
					JsonParser parser = new JsonParser();
					jsonObject = (JsonObject) parser.parse(reader);
				} catch (JsonParseException ex) {
					logger.warn(ex.toString());
				} finally {
					reader.close();
					infoConn.disconnect();
				}

				if (jsonObject == null) {
					continue;
				}

				episode.setUrl(href);

				JsonElement description = jsonObject.get("description");
				episode.setIntroduction(description == null || description.isJsonNull() ? "" : description
						.getAsString());

				JsonElement title = jsonObject.get("title");
				if (title == null) {
					title = jsonObject.get("show_name");
				}
				episode.setEpisodeName(title == null || title.isJsonNull() ? "" : title.getAsString());

				JsonElement thumbnailUrl = jsonObject.get("thumbnail_url");
				episode.setThumbnail(thumbnailUrl == null || thumbnailUrl.isJsonNull() ? "" : thumbnailUrl.getAsString());

				JsonElement score = jsonObject.get("rating");
				episode.setScore(score == null || score.isJsonNull() ? BigDecimal.ZERO : score.getAsBigDecimal());

				JsonElement publishTime = jsonObject.get("air_date");
				String time = publishTime == null || publishTime.isJsonNull() ? "" : publishTime.getAsString();
				if ( ! "".equals(time)) {					
					episode.setAirDate(DateUtils.parse(airDateFormat, time));
				}
				
				JsonElement duration = jsonObject.get("duration");
				episode.setDuration(duration == null || duration.isJsonNull() ? 0 : duration.getAsInt());

				JsonElement part = jsonObject.get("episode_number");
				episode.setPart(part == null || part.isJsonNull() ? 1 : part.getAsInt());

				JsonElement season = jsonObject.get("season_number");
				episode.setSeason(season == null || season.isJsonNull() ? 1 : season.getAsInt());

				// show_id + season + part
				episode.setId(episode.getShowId() + "E" + episode.getSeason() + "E" + episode.getPart());
				
				listResult.add(episode);
			}
		}

		return listResult;
	}

	private static List<Movie> obtainMovies(final Document doc, final String movieType) throws IOException {
		List<Movie> listResult = new ArrayList<Movie>();
		final SimpleDateFormat airDateFormat = new SimpleDateFormat("MM/dd/yyyy");
		
		Elements elements = doc.select("#results_list > tbody tr > td ");
		for (int i = 0; i < elements.size(); i++) {
			Element td = elements.get(i);
			Elements link = td.select("div.home-play-container > span.play-button-hover > a").eq(0);
			String beaconid = link.attr("beaconid");
			if ( ! "".equals(beaconid)) {
				String itemUrl = link.attr("href");
				String infoUrl = "http://www.hulu.com/videos/info/" + beaconid;
				HttpURLConnection infoConn = (HttpURLConnection) new URL(infoUrl).openConnection();
				infoConn.connect();

				JsonObject jsonObject = null;
				InputStreamReader reader = new InputStreamReader(infoConn.getInputStream(), AppEnvironment.charsetName);
				try {
					JsonParser parser = new JsonParser();
					jsonObject = (JsonObject) parser.parse(reader);
				} catch (JsonParseException ex) {					
					logger.warn(ex.toString());
				} finally {
					reader.close();
					infoConn.disconnect();
				}

				if (jsonObject != null) {
					Movie item = new Movie();					
					item.setWebsite(SiteConstants.WEBSITE_HULU);
					item.setArea(StandarAreas.AMERICA);
					item.setMovieType(movieType);
					item.setAllowEmbed(0);
					item.setItemUrl(itemUrl);

					JsonElement description = jsonObject.get("description");
					item.setDescription(description == null || description.isJsonNull() ? "" : description
							.getAsString());

					JsonElement title = jsonObject.get("title");
					item.setTitle(title == null || title.isJsonNull() ? "" : title.getAsString());
					//首字母
					item.setTitleInitial(StringUtils.getInitial(item.getTitle()));
					
					JsonElement parentTitle = jsonObject.get("show_name");
					String strParentTitle = parentTitle == null || parentTitle.isJsonNull() ? "" : parentTitle.getAsString();
					if ( ! StringUtils.isEqual(strParentTitle, item.getTitle())) {
						item.setParentTitle(strParentTitle);
					}

					JsonElement thumbnailUrl = jsonObject.get("thumbnail_url");
					item.setThumbnail(thumbnailUrl == null || thumbnailUrl.isJsonNull() ? "" : thumbnailUrl.getAsString());

					JsonElement id = jsonObject.get("id");
					item.setVideoId(id == null || id.isJsonNull() ? "" : id.getAsString());
					
					// "hl" + videoId
					item.setId(SiteConstants.ID_PREFIX_HULU + item.getVideoId());
					
					JsonElement score = jsonObject.get("rating");
					item.setScore(score == null || score.isJsonNull() ? BigDecimal.ZERO : score.getAsBigDecimal());

					JsonElement publishTime = jsonObject.get("air_date");
					String time = publishTime == null || publishTime.isJsonNull() ? "" : publishTime	.getAsString();
					
					if ( ! "".equals(time)) {						
						item.setPublishTime( DateUtils.parse(airDateFormat, time));
					}
					
					JsonElement duration = jsonObject.get("duration");
					item.setDuration(duration == null || duration.isJsonNull() ? 0 : duration.getAsInt());

					Elements links = td.select("div.home-thumb div.mores-div span.video-info > a");
					if (links.size() > 0) {
						// games 中， 最后一个才是真正的genre
						String genre = links.last().text();
						item.setGenre(genre);
					}
					
					listResult.add(item);
				}
			}
		}

		return listResult;
	}
	
	private static Document getDocument(String url){
		try {
			return Jsoup.connect(url).referrer(REFERER).userAgent(USER_AGENT).timeout(TIMEOUT).get();
		} catch (Exception e) {			
			logger.error(e.toString());
		}
		return null;
	}
}
